# -*- coding: utf-8 -*-
from micolog_plugin import *
from model import *
import re,datetime
from google.appengine.api import urlfetch
from xml.dom import minidom
from HTMLParser import HTMLParser

HEADERS = { 'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.0; zh-CN; rv:1.9.1.9) Gecko/20100315 Firefox/3.5.9'}

class FeedList(db.Model):
    author = db.StringProperty(multiline=False,default='robot')
    name = db.StringProperty(multiline=False,default='Fred')
    feedurl = db.StringProperty(multiline=False,default='http://hi.baidu.com/429263181/rss')
    latest = db.StringProperty(multiline=False,default='last')
    abconf = db.StringListProperty()
    repReg = db.StringProperty(multiline=False)
    autoUrl = db.BooleanProperty(default=False)
    autoTimeFormat = db.StringProperty(multiline=False,default=None)
    allow_comment = db.BooleanProperty(default=True) #allow comment

class pick_rss(Plugin):
    def __init__(self):
        Plugin.__init__(self,__file__)
        self.author="Fred"
        self.authoruri="mailto:ad@fengsage.cn"
        self.uri="http://fengsageblog.appspot.com"
        self.description="RSS采集"
        self.name="PickRss"
        self.version="0.2"
        self.register_urlmap('pick',self.getFeed)

    #RSS列表、删除RSS
    def get(self,page):
        if page.param("delid")=='':
            listit = FeedList()
            querys = listit.all()
            return self.render_content("pick_rss.html",{'list':querys})
        else:
            #删除
            listit = FeedList()
            querys=listit.all().filter('name =', page.param("delid")).fetch(1)
            for query in querys:
                query.delete()
                return "Delete it successfully! <a href='?'>Click here BACK</a>"

    #新增RSS
    def post(self,page):
        query = FeedList()
        query.name =page.param("name")
        query.feedurl = page.param("feedurl")
        query.abconf = page.request.get_all('abconf')
        query.autoTimeFormat = page.param("autoTimeFormat")

        logging.info(query.abconf)

        if page.param("allowComment") == 'on':
            query.allow_comment = True
        else:
            query.allow_comment = False

        if page.param("autoUrl") == 'on':
            query.autoUrl = True
        else:
            query.autoUrl = False

        if page.param("repReg"):
            query.repReg = page.param("repReg")
            
        query.put()
        return self.get(page)

    def getFeed(self,page=None,*arg1,**arg2):
        listit = FeedList()
        querys = listit.all()
        #遍历数据库记录数据
        for detal in querys:
            #rss adr
            url=str(detal.feedurl)
#            logging.info("**"+url);

            #读取RSS连接,模拟浏览器

            result = urlfetch.fetch(url,None,urlfetch.GET,HEADERS)

#            logging.info(rss_source)
            if result.status_code == 200:
                rss_source = result.content
                #编码转换
                if 'encoding="gb2312"' in rss_source:
                    rss_source = result.content.decode('gb2312').encode('UTF-8')
                    rss_source = rss_source.replace('encoding="gb2312"','encoding="utf-8"')
                if 'encoding="GBK"' in rss_source:
                    rss_source = result.content.decode('GBK').encode('UTF-8')
                    rss_source = rss_source.replace('encoding="GBK"','encoding="utf-8"')
                #解析XML开始
                file_xml = minidom.parseString(rss_source)
                rssNOs=file_xml.getElementsByTagName('rss')
                rssver='1.0'

                #判断RSS版本
                for rssNO in rssNOs:
                    rssver=rssNO.getAttribute('version')
                if rssver=='1.0':
                    artList='entry'
                    artTitle='title'
                    artLink='link'
                    artText='content'
                    artTime='date'
                else:
                    artList='item'
                    artTitle='title'
                    artLink='link'
                    artText='description'
                    artTime='pubDate'


                #获得RSS内所有文章根节点
                items = file_xml.getElementsByTagName(artList)

                flag=''
                latestId=detal.latest
                ifFirst=0

                #按顺序遍历插入数据,默认情况下与录入数据时间与现实相反的
                #可以开启同步时间,保持远程RSS与本地数据时间同步
                for item in items:
                    entry=Entry()
                    entry.title=item.getElementsByTagName(artTitle)[0].firstChild.data
                    entry.author_name = detal.author

                    #自动构建URL（同步GoogleTranslate）
                    if detal.autoUrl:
                        entry.slug = autoSlug(entry.title)

                    #自动时间同步
                    if detal.autoTimeFormat:
                        #格式化时间
                        format = detal.autoTimeFormat
                        article_date = item.getElementsByTagName(artTime)[0].firstChild.data
                        entry.date = formatDate(article_date, format)

                    #评论是否开启
                    if detal.allow_comment:
                        entry.allow_comment = True
                    else:
                        entry.allow_comment = False

                    if rssver=='1.0':
                        flag=item.getElementsByTagName(artLink)[0].getAttribute('href')
                    else:
                        flag=item.getElementsByTagName(artLink)[0].firstChild.data

                    #保存最后一条记录，防止重复采集
                    if latestId=='last':
                        detal.latest=flag
                        latestId=flag
                        ifFirst=1
                        db.put(detal)
                    else:
                        if flag==latestId:
                            break
                        else:
                            if ifFirst==0:
                                detal.latest=flag
                                db.put(detal)
                                ifFirst=1
                        db.put(detal)

                    artContent=item.getElementsByTagName(artText)[0].firstChild.data
#                    logging.info(artContent)
                    #过滤内容
                    for abc in detal.abconf:
                     #过滤所有a标签
                        if abc=='0' :
                            artContent = re.sub("<a([^>]*?)>","",artContent)
                            artContent = re.sub("</a>","",artContent)
                        #过滤所有img标签
                        if abc=='1' :
                            artContent = re.sub("<img(.*?>)","",artContent)
                        #根据表达式替换文本
                        if abc=='2':
                            if detal.repReg:
                                for txt in detal.repReg.split(','):
                                    rep = txt.split(':')
                                    artContent = re.sub(rep[0],rep[1],artContent)
  
                    #替换所有换行符
                    artContent=artContent.replace('\n', "<br />")
                    entry.content=artContent
                    entry.save(True)



def formatDate(str,format):
    return datetime.datetime.strptime(str,format)

def autoSlug(str):
    translate_url = 'http://translate.google.com/translate_a/t?client=t&text=%s&hl=en&sl=zh-CN&tl=en&pc=0'%str.replace(' ','-').encode("utf-8")
    translate_result = urlfetch.fetch(translate_url,None,urlfetch.GET,HEADERS)

    if translate_result.status_code == 200:
        trans_reg = u'"trans":"([^"]*)"'
        translate_content = re.findall(trans_reg, translate_result.content)[0]

        translate_content = translate_content   \
                        .replace(' ','-')   \
                        .replace('\\','-')

        translate_content = re.sub(u'[^a-zA-Z\d_]','-', translate_content)

        logging.info("*********"+translate_content)
        return translate_content



if __name__=="__main__":
    starttime = '2010-05-15  14:14'
    format="%Y-%m-%d %H:%M";
    print datetime.datetime.strptime(starttime,format)



    